from internal_scripts.data_loaders.BlackFridayDataLoader import BlackFridayDataLoader
from internal_scripts.data_loaders.LoanDataLoader import *
from internal_scripts.modelling.loading.models_loading import get_saved_models
from internal_scripts.descriptors.Eli5Descriptor import Eli5Descriptor
from internal_scripts.descriptors.ShapDescriptor import ShapDescriptor
from internal_scripts.descriptors.LimeDescriptor import LimeDescriptor
import shap
shap.initjs()
from IPython.display import display, HTML
loaders = [LoanDataLoader(), BlackFridayDataLoader()]
descriptors = [Eli5Descriptor(), ShapDescriptor(), LimeDescriptor()]
final_dict = {}
for loader in loaders:
data = loader.get_train_test_split()
data_name = data['dataset_name']
print(f"Working with {data_name}")
models = get_saved_models(data_name)
final_dict[data_name] = {}
for name, model in models.items():
print(f"Describing {name} model")
final_dict[data_name][name] = {}
for desc in descriptors:
desc_name = desc.get_descriptor_name()
print(f"Using {desc_name} descriptor")
model_desc = desc.describe(name, model, data)
final_dict[data_name][name][desc_name] = model_desc
print()
Working with Loan_Data
Using TensorFlow backend.
Describing Decision_Tree_Loan_Data model Using Eli 5 descriptor Using Shap descriptor Using Lime descriptor Decision_Tree_Loan_Data Intercept 0.5890724168765763 Prediction_local [0.61193332] Right: 0.0 Describing Logistic_Regression_Loan_Data model Using Eli 5 descriptor Using Shap descriptor Model type not yet supported by TreeExplainer: <class 'sklearn.linear_model.logistic.LogisticRegression'> Using Lime descriptor Logistic_Regression_Loan_Data Describing Random_Forest_Classifier_Loan_Data model Using Eli 5 descriptor Using Shap descriptor Using Lime descriptor Random_Forest_Classifier_Loan_Data Describing XGboost_Loan_Data model Using Eli 5 descriptor Using Shap descriptor Using Lime descriptor XGboost_Loan_Data Describing Keras_Simple_Classifier_Loan_Data model Using Eli 5 descriptor Using Shap descriptor
c:\program files\python36\lib\site-packages\shap\explainers\deep\deep_tf.py:138: UserWarning: You have provided over 5k background samples! For better performance consider using smaller random sample.
warnings.warn("You have provided over 5k background samples! For better performance consider using smaller random sample.")
Using Lime descriptor Keras_Simple_Classifier_Loan_Data Working with Black_Friday Describing Decision_Tree_Black_Friday model Using Eli 5 descriptor Using Shap descriptor Using Lime descriptor Decision_Tree_Black_Friday
c:\program files\python36\lib\site-packages\sklearn\utils\validation.py:595: DataConversionWarning: Data with input dtype int64 was converted to float64 by StandardScaler. warnings.warn(msg, DataConversionWarning)
Intercept 0.24586254168401653 Prediction_local [0.43496527] Right: 0.3333333333333333 Describing Logistic_Regression_Black_Friday model Using Eli 5 descriptor Using Shap descriptor Model type not yet supported by TreeExplainer: <class 'sklearn.linear_model.logistic.LogisticRegression'> Using Lime descriptor Logistic_Regression_Black_Friday Describing Random_Forest_Classifier_Black_Friday model Using Eli 5 descriptor Using Shap descriptor Using Lime descriptor Random_Forest_Classifier_Black_Friday Describing XGboost_Black_Friday model Using Eli 5 descriptor Using Shap descriptor Using Lime descriptor XGboost_Black_Friday Describing Keras_Simple_Classifier_Black_Friday model Using Eli 5 descriptor Using Shap descriptor Using Lime descriptor Keras_Simple_Classifier_Black_Friday
for dataset, dicts in final_dict.items():
display(HTML(f"<h1>{dataset} data</h1>"))
for model, results in dicts.items():
display(HTML(f"<h2>{model} data</h2>"))
for descriptor, results in results.items():
if len(results) == 0:
continue
display(HTML(f"<h3>{descriptor} data</h3>"))
for metric, result in results.items():
display(HTML(f"<h4>{metric} data</h4>"))
display(result)
| Weight | Feature |
|---|---|
| 0.2222 | Credit_Score |
| 0.1223 | Current_Loan_Amount |
| 0.0941 | Monthly_Debt |
| 0.0882 | Years_of_Credit_History |
| 0.0850 | Maximum_Open_Credit |
| 0.0792 | Current_Credit_Balance |
| 0.0716 | Annual_Income |
| 0.0545 | Number_of_Open_Accounts |
| 0.0454 | Months_since_last_delinquent |
| 0.0087 | Years_in_current_job_10more_years |
| 0.0072 | Years_in_current_job_6_years |
| 0.0069 | Number_of_Credit_Problems |
| 0.0067 | Bankruptcies |
| 0.0066 | Home_Ownership_Rent |
| 0.0060 | Years_in_current_job_8_years |
| 0.0060 | Years_in_current_job_2_years |
| 0.0060 | Years_in_current_job_5_years |
| 0.0059 | Years_in_current_job_3_years |
| 0.0058 | Home_Ownership_Home_Mortgage |
| 0.0058 | Years_in_current_job_less_1_year |
| … 26 more … | |
y=0 (probability 1.000) top features
| Contribution? | Feature |
|---|---|
| +0.667 | Purpose_Debt_Consolidation |
| +0.302 | Current_Loan_Amount |
| +0.226 | <BIAS> |
| +0.035 | Home_Ownership_Home_Mortgage |
| +0.027 | Credit_Score |
| -0.007 | Current_Credit_Balance |
| -0.023 | Number_of_Open_Accounts |
| -0.041 | Term_Short_Term |
| -0.049 | Maximum_Open_Credit |
| -0.060 | Monthly_Debt |
| -0.077 | Years_of_Credit_History |
y=1 top features
| Weight? | Feature |
|---|---|
| +0.000 | Monthly_Debt |
| +0.000 | Months_since_last_delinquent |
| +0.000 | Years_of_Credit_History |
| +0.000 | Annual_Income |
| +0.000 | Number_of_Open_Accounts |
| +0.000 | Current_Loan_Amount |
| +0.000 | Maximum_Open_Credit |
| +0.000 | Term_Short_Term |
| +0.000 | <BIAS> |
| +0.000 | Purpose_Debt_Consolidation |
| +0.000 | Home_Ownership_Home_Mortgage |
| +0.000 | Home_Ownership_Rent |
| +0.000 | Years_in_current_job_10more_years |
| +0.000 | Number_of_Credit_Problems |
| +0.000 | Bankruptcies |
| +0.000 | Years_in_current_job_3_years |
| +0.000 | Years_in_current_job_2_years |
| … 22 more positive … | |
| … 5 more negative … | |
| -0.000 | Term_Long_Term |
| -0.000 | Current_Credit_Balance |
| -0.000 | Credit_Score |
y=1 (probability 0.635, score 0.276) top features
| Contribution? | Feature |
|---|---|
| +0.181 | Monthly_Debt |
| +0.106 | Maximum_Open_Credit |
| +0.023 | Current_Loan_Amount |
| +0.000 | Credit_Score |
| +0.000 | Years_of_Credit_History |
| +0.000 | Months_since_last_delinquent |
| +0.000 | Number_of_Open_Accounts |
| +0.000 | Term_Short_Term |
| +0.000 | <BIAS> |
| +0.000 | Purpose_Debt_Consolidation |
| +0.000 | Home_Ownership_Rent |
| +0.000 | Years_in_current_job_10more_years |
| -0.000 | Annual_Income |
| -0.034 | Current_Credit_Balance |
| Weight | Feature |
|---|---|
| 0.2120 ± 0.0248 | Credit_Score |
| 0.1139 ± 0.0176 | Current_Loan_Amount |
| 0.0894 ± 0.0053 | Monthly_Debt |
| 0.0894 ± 0.0069 | Maximum_Open_Credit |
| 0.0877 ± 0.0084 | Current_Credit_Balance |
| 0.0835 ± 0.0073 | Years_of_Credit_History |
| 0.0748 ± 0.0086 | Annual_Income |
| 0.0629 ± 0.0067 | Number_of_Open_Accounts |
| 0.0471 ± 0.0059 | Months_since_last_delinquent |
| 0.0090 ± 0.0029 | Number_of_Credit_Problems |
| 0.0081 ± 0.0064 | Years_in_current_job_10more_years |
| 0.0078 ± 0.0023 | Bankruptcies |
| 0.0071 ± 0.0072 | Purpose_Debt_Consolidation |
| 0.0069 ± 0.0080 | Term_Long_Term |
| 0.0063 ± 0.0014 | Years_in_current_job_less_1_year |
| 0.0063 ± 0.0022 | Years_in_current_job_2_years |
| 0.0058 ± 0.0029 | Home_Ownership_Rent |
| 0.0056 ± 0.0016 | Years_in_current_job_5_years |
| 0.0054 ± 0.0013 | Years_in_current_job_1_year |
| 0.0052 ± 0.0011 | Years_in_current_job_4_years |
| … 26 more … | |
y=0 (probability 0.900) top features
| Contribution? | Feature |
|---|---|
| +0.339 | Current_Loan_Amount |
| +0.227 | <BIAS> |
| +0.134 | Current_Credit_Balance |
| +0.133 | Years_of_Credit_History |
| +0.058 | Months_since_last_delinquent |
| +0.056 | Maximum_Open_Credit |
| +0.041 | Monthly_Debt |
| +0.013 | Home_Ownership_Home_Mortgage |
| +0.011 | Annual_Income |
| +0.001 | Years_in_current_job_less_1_year |
| +0.001 | Years_in_current_job_7_years |
| +0.001 | Years_in_current_job_6_years |
| +0.000 | Purpose_Home_Improvements |
| -0.000 | Bankruptcies |
| -0.000 | Purpose_Business_Loan |
| -0.000 | Purpose_moving |
| -0.000 | Tax_Liens |
| -0.000 | Purpose_small_business |
| -0.000 | Purpose_Buy_a_Car |
| -0.000 | Purpose_Other |
| -0.001 | Purpose_Debt_Consolidation |
| -0.001 | Number_of_Credit_Problems |
| -0.001 | Years_in_current_job_8_years |
| -0.001 | Home_Ownership_Own_Home |
| -0.002 | Purpose_other |
| -0.003 | Years_in_current_job_4_years |
| -0.003 | Years_in_current_job_2_years |
| -0.003 | Home_Ownership_Rent |
| -0.003 | Years_in_current_job_-1 |
| -0.005 | Years_in_current_job_10more_years |
| -0.005 | Years_in_current_job_3_years |
| -0.008 | Years_in_current_job_5_years |
| -0.009 | Number_of_Open_Accounts |
| -0.013 | Term_Short_Term |
| -0.023 | Credit_Score |
| -0.034 | Term_Long_Term |
| Weight | Feature |
|---|---|
| 0.3701 | Credit_Score |
| 0.1750 | Current_Loan_Amount |
| 0.1037 | Term_Long_Term |
| 0.0528 | Annual_Income |
| 0.0454 | Home_Ownership_Rent |
| 0.0409 | Home_Ownership_Home_Mortgage |
| 0.0305 | Years_in_current_job_-1 |
| 0.0210 | Current_Credit_Balance |
| 0.0196 | Monthly_Debt |
| 0.0169 | Maximum_Open_Credit |
| 0.0163 | Purpose_Business_Loan |
| 0.0160 | Purpose_small_business |
| 0.0152 | Number_of_Credit_Problems |
| 0.0144 | Years_in_current_job_3_years |
| 0.0136 | Purpose_other |
| 0.0133 | Years_of_Credit_History |
| 0.0129 | Years_in_current_job_6_years |
| 0.0107 | Number_of_Open_Accounts |
| 0.0087 | Months_since_last_delinquent |
| 0.0031 | Years_in_current_job_less_1_year |
| … 26 more … | |
y=1 (probability 0.758, score 1.140) top features
| Contribution? | Feature |
|---|---|
| +1.321 | <BIAS> |
| +0.135 | Term_Long_Term |
| +0.075 | Maximum_Open_Credit |
| +0.036 | Current_Credit_Balance |
| +0.013 | Years_in_current_job_-1 |
| +0.011 | Current_Loan_Amount |
| +0.008 | Number_of_Open_Accounts |
| +0.006 | Years_of_Credit_History |
| +0.003 | Purpose_other |
| +0.002 | Purpose_Business_Loan |
| +0.002 | Purpose_small_business |
| -0.002 | Years_in_current_job_3_years |
| -0.014 | Credit_Score |
| -0.020 | Months_since_last_delinquent |
| -0.042 | Home_Ownership_Rent |
| -0.044 | Home_Ownership_Home_Mortgage |
| -0.069 | Monthly_Debt |
| -0.279 | Annual_Income |
| Weight | Feature |
|---|---|
| 0.3341 | Occupation |
| 0.1071 | Marital_Status |
| 0.1040 | City_Category_C |
| 0.0523 | Stay_In_Current_City_Years_2 |
| 0.0519 | Stay_In_Current_City_Years_1 |
| 0.0425 | Age_46-50 |
| 0.0380 | Stay_In_Current_City_Years_4+ |
| 0.0366 | Stay_In_Current_City_Years_3 |
| 0.0353 | Stay_In_Current_City_Years_0 |
| 0.0351 | Age_18-25 |
| 0.0258 | Age_51-55 |
| 0.0251 | Age_26-35 |
| 0.0240 | Age_36-45 |
| 0.0203 | City_Category_A |
| 0.0194 | City_Category_B |
| 0.0184 | Age_55+ |
| 0.0150 | Age_0-17 |
| 0.0148 | Gender_F |
| y=0 (probability 0.667) top features | y=1 (probability 0.333) top features | y=2 (probability 0.000) top features | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
|
|
| y=0 top features | y=1 top features | y=2 top features | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
|
|
| y=0 (probability 0.411, score 0.268) top features | y=1 (probability 0.399, score 0.239) top features | y=2 (probability 0.189, score -0.507) top features | ||||||||||||||||||||||||||||||||||||||||||
|
|
|
| Weight | Feature |
|---|---|
| 0.5291 ± 0.1273 | Occupation |
| 0.0691 ± 0.0384 | Marital_Status |
| 0.0514 ± 0.0410 | Gender_F |
| 0.0488 ± 0.0459 | City_Category_C |
| 0.0354 ± 0.0366 | City_Category_B |
| 0.0268 ± 0.0127 | Stay_In_Current_City_Years_1 |
| 0.0247 ± 0.0127 | Stay_In_Current_City_Years_2 |
| 0.0244 ± 0.0161 | Stay_In_Current_City_Years_4+ |
| 0.0243 ± 0.0255 | Age_26-35 |
| 0.0239 ± 0.0169 | Stay_In_Current_City_Years_3 |
| 0.0234 ± 0.0122 | Age_18-25 |
| 0.0221 ± 0.0182 | Stay_In_Current_City_Years_0 |
| 0.0207 ± 0.0212 | Age_36-45 |
| 0.0203 ± 0.0115 | Age_51-55 |
| 0.0191 ± 0.0123 | Age_46-50 |
| 0.0164 ± 0.0081 | Age_55+ |
| 0.0107 ± 0.0113 | City_Category_A |
| 0.0094 ± 0.0038 | Age_0-17 |
| y=0 (probability 0.498) top features | y=1 (probability 0.502) top features | y=2 (probability 0.000) top features | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
|
|
| Weight | Feature |
|---|---|
| 0.5134 | City_Category_C |
| 0.0589 | Gender_F |
| 0.0560 | Age_55+ |
| 0.0422 | City_Category_B |
| 0.0345 | Stay_In_Current_City_Years_3 |
| 0.0309 | Age_26-35 |
| 0.0285 | Stay_In_Current_City_Years_1 |
| 0.0278 | Age_51-55 |
| 0.0268 | City_Category_A |
| 0.0249 | Age_36-45 |
| 0.0241 | Marital_Status |
| 0.0230 | Occupation |
| 0.0209 | Stay_In_Current_City_Years_0 |
| 0.0203 | Stay_In_Current_City_Years_4+ |
| 0.0180 | Age_0-17 |
| 0.0174 | Age_18-25 |
| 0.0170 | Age_46-50 |
| 0.0154 | Stay_In_Current_City_Years_2 |
| y=0 (probability 0.415, score 0.332) top features | y=1 (probability 0.434, score 0.377) top features | y=2 (probability 0.151, score -0.679) top features | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
|
|